package com.qen.yanxuan

import cn.hutool.core.date.DateUtil
import cn.hutool.core.lang.Console
import cn.hutool.poi.excel.ExcelUtil
import com.qen.yanshen.THREAD_COUNT
import com.qen.yanshen.UserAgent
import it.skrape.core.htmlDocument
import it.skrape.fetcher.HttpFetcher
import it.skrape.fetcher.Method
import it.skrape.fetcher.response
import it.skrape.fetcher.skrape
import java.util.Date
import java.util.concurrent.CountDownLatch
import java.util.concurrent.atomic.AtomicInteger
import kotlin.collections.forEach

const val YanXuanUrl = "https://www.yanxuanwk.com/topic/yanxuan/"
const val YanXuanMaxPage = 930
const val YanxuanOutFile = "/Users/a1/Documents/Git/Kotlin/Eleven/out/yanxuan"
fun main() {

    /**爬虫线程队列*/
    val threads = ArrayList<Thread>()

    /**已爬取的新闻数（所谓“已爬取”，实际是从拿到链接就算）*/
    val pageNum = AtomicInteger(1)

    /**爬取结束信号*/
    val newsTargetReachedSemaphore = CountDownLatch(THREAD_COUNT)

    /**待爬取队列非空信号*/
//    val linkQueueAvailableSemaphore = Semaphore(Int.MAX_VALUE, Int.MAX_VALUE)

    /**放出一个信号*/
//    linkQueueAvailableSemaphore.release()

    var referer = YanXuanUrl
    val list: MutableList<YanxuanArticle> = ArrayList<YanxuanArticle>()
    for (i in 0..THREAD_COUNT) {
        val thread = Thread {
            try {

                while (true) {

                    val urlNew = if (pageNum.get() == 1) YanXuanUrl else YanXuanUrl + "page/${pageNum.get()}/"

//                    runBlocking {
//                        linkQueueAvailableSemaphore.acquire()
//                    }
                    if (pageNum.get() >= YanXuanMaxPage) {
                        break
                    }

                    Console.log("${Thread.currentThread().threadId()}：pageNum:${pageNum.get()}")
                    try {
                        list.addAll(articleUrl(urlNew, referer, pageNum.get()))
                    } catch (e: Exception) {
                        e.printStackTrace()
                    }

                    val newsCount = pageNum.getAndIncrement()
                    if (newsCount >= YanXuanMaxPage) { //如果拿到目标数就结束
                        newsTargetReachedSemaphore.countDown()
                    }
//                    linkQueueAvailableSemaphore.release()
                    referer = urlNew
                }
                newsTargetReachedSemaphore.countDown()
            } catch (e: Exception) {
                e.printStackTrace()
            }
        }
        thread.start()
        threads.add(thread)
    }

    newsTargetReachedSemaphore.await()

    threads.forEach { t -> t.join() }

    Console.error("多线程结束了")
    val writer = ExcelUtil.getWriter(
        YanxuanOutFile + "/${DateUtil.format(Date(), "YYYYMMdd")}/Yanxuan-${
            DateUtil.format(
                Date(),
                "YYYYMMdd"
            )
        }.xlsx"
    )
    writer.passCurrentRow()
//    writer.merge(quoteList.size - 1, "标题")
    writer.addHeaderAlias("title", "标题");
    writer.addHeaderAlias("time", "时间");
    writer.addHeaderAlias("url", "链接");
    writer.write(list, true)
    writer.close()
}

data class YanxuanArticle(
    var title: String,
    var url: String,
    var time: String
){
}

fun initYanXuanHeader(): Map<String, String> {
    return mutableMapOf(
        "accept" to "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "accept-language" to "zh-CN,zh;q=0.9,en;q=0.8",
        "cache-control" to "no-cache",
        "pragma" to "no-cache",
        "priority" to "u=0, i",
        "sec-ch-ua" to "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\"",
        "sec-ch-ua-mobile" to "?0",
        "sec-ch-ua-platform" to "\"macOS\"",
        "sec-fetch-dest" to "document",
        "sec-fetch-mode" to "navigate",
        "sec-fetch-site" to "same-origin",
        "sec-fetch-user" to "?1",
        "upgrade-insecure-requests" to "1"
    )
}

fun initYanXuanCookie(): Map<String, String> {
    return mutableMapOf(
        "_ga" to "GA1.1.1256512866.1729579340",
        "fontSize" to "18.00",
        "_ga_B79H5MXFYF" to "GS1.1.1729579339.1.1.1729581094.0.0.0"
    )
}

fun articleUrl(goUrl: String, refererUrl: String, i: Int): ArrayList<YanxuanArticle> {
    val list = ArrayList<YanxuanArticle>()
    var topurl = if (i == 1) "/topic/yanxuan/" else "/topic/yanxuan/page/$i/"
    skrape(HttpFetcher) {
        request {
            url = goUrl
            headers = initYanXuanHeader()
            cookies = initYanXuanCookie()
            userAgent = UserAgent
            method = Method.GET
            body = null
//            proxy = ProxyBuilder(Proxy.Type.HTTP, "127.0.0.1", 26001)
        }
        response {
            htmlDocument {
                ".entry-header" {
                    findAll {
                        forEach { t ->
                            val ch = t.findFirst(".entry-title").children
                            val a = ch.get(0)
                            val tm = t.findFirst(".entry-meta").findFirst(".meta-date").findFirst(".updated")
                            Console.log("title:${a.text};url:${a.attribute("href")}; time:${tm.text}")
                            list.add(
                                YanxuanArticle(
                                    title = a.text,
                                    url = a.attribute("href"),
                                    time = tm.text
                                )
                            )
                        }
                    }
                }
            }
        }
    }
    return list
}

fun main1() {
    val list =
        articleUrl(
            goUrl = "https://www.yanxuanwk.com/topic/yanxuan/page/2/",
            refererUrl = "https://www.yanxuanwk.com/topic/yanxuan/",
            i = 2
        )
    Console.log(list.joinToString("\n"))
}
